{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "基于内容的分析是对用户的基本信息和电影的基本信息进行标识和匹配\n",
    "\n",
    "用户基本信息表：u.user 共5个字段： 1）user_id：用户ID 2）age：年龄 3）gender：性别 4）occupation：职业 5）zip_code：邮编<br>\n",
    "电影基本信息表：u.item 共5 + 19 个字段： 1）movie_id：电影ID 2）title：电影标题（带年份） 3）release_date：电影发布日期 4）video_release_date：Video发布日期 5）imdb_url：链接"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-17T02:43:52.586458Z",
     "start_time": "2020-09-17T02:43:52.581458Z"
    }
   },
   "source": [
    "对训练数据：1.建立用户和物品索引，方便用下标访问打分表；2.建立倒排表，加速查询访问；3.打分表按用户、物品索引保存为稀疏矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 协同过滤数据的基本处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T11:33:03.493344Z",
     "start_time": "2020-08-26T11:33:01.787324Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "#字典 用于建立用户和item的索引 \n",
    "from collections import defaultdict\n",
    "\n",
    "#稀疏矩阵，存储打分表\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "\n",
    "#数据到文件储存 \n",
    "import pickle as pk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T11:33:03.626843Z",
     "start_time": "2020-08-26T11:33:03.495081Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>874965758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>876893171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>878542960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>876893119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>889751712</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0        1        1       5  874965758\n",
       "1        1        2       3  876893171\n",
       "2        1        3       4  878542960\n",
       "3        1        4       3  876893119\n",
       "4        1        5       3  889751712"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取训练数据\n",
    "triplet_cols = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "df_triplet = pd.read_csv(\"u1.base\", sep = '\\t', names = triplet_cols, encoding = 'latin - 1')\n",
    "df_triplet.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T11:33:03.634352Z",
     "start_time": "2020-08-26T11:33:03.627795Z"
    }
   },
   "outputs": [],
   "source": [
    "#统计user和item数目\n",
    "unique_users = df_triplet['user_id'].unique()\n",
    "unique_items = df_triplet['item_id'].unique()\n",
    "\n",
    "n_users = unique_users.shape[0]\n",
    "n_items = unique_items.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T11:33:03.645749Z",
     "start_time": "2020-08-26T11:33:03.636079Z"
    }
   },
   "outputs": [],
   "source": [
    "#建立用户和item的索引\n",
    "#本数据集中user_id and item_id都已经是索引了，可以减1，将从1开始编码变为从0开始编码\n",
    "users_index = dict()\n",
    "items_index = dict()\n",
    "\n",
    "for j, u in enumerate(unique_users):\n",
    "    users_index[u] = j \n",
    "    \n",
    "#重新编码活动索引字典     \n",
    "for j, i in enumerate(unique_items):\n",
    "    items_index[i] = j\n",
    "    \n",
    "#保存用户和item的索引\n",
    "pk.dump(users_index, open(\"user_index.pkl\", 'wb'))\n",
    "pk.dump(users_index, open(\"item_index.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-26T11:33:18.694804Z",
     "start_time": "2020-08-26T11:33:03.647742Z"
    }
   },
   "outputs": [],
   "source": [
    "#建立倒排表（统计每个用户打过分的电影，以及每个电影被哪些用户打过分）\n",
    "user_items = defaultdict(set)\n",
    "item_users = defaultdict(set)\n",
    "\n",
    "#用户-item关系矩阵R,稀疏矩阵，记录每个用户对电影的打分\n",
    "user_item_scores = ss.dok_matrix((n_users, n_items))\n",
    "\n",
    "#扫描训练数据\n",
    "for line in df_triplet.index:#对每条记录\n",
    "    cur_user_index = users_index [df_triplet.iloc[line]['user_id']]\n",
    "    cur_item_index = items_index [df_triplet.iloc[line]['item_id']]\n",
    "    \n",
    "    #到排序\n",
    "    user_items[cur_user_index].add(cur_item_index)#该用户对这个电影进行了打分\n",
    "    item_users[cur_item_index].add(cur_user_index)#该电影被改用户打分\n",
    "    \n",
    "    user_item_scores[cur_user_index, cur_item_index] = df_triplet.iloc[line]['rating']\n",
    "    \n",
    "##保存倒排表\n",
    "#每个用户打分的电影\n",
    "pk.dump(user_items, open(\"user_items.pkl\", 'wb'))\n",
    "#对每个电影打分的用户\n",
    "pk.dump(item_users, open(\"item_users.pkl\", 'wb'))\n",
    "#保存打分矩阵，在基于用户的协同过滤和itemCF中会用到\n",
    "sio.mmwrite(\"user_item_scores\", user_item_scores)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
